package edu.hit.pact.pgse.crawler.bbs;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.logging.LogFactory;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;


import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.bean.RunnableTask;
import edu.hit.pact.pgse.crawler.Crawler;
import edu.hit.pact.pgse.crawler.download.WebPageFetcher;
import edu.hit.pact.pgse.crawler.finish.FinishChecker;
import edu.hit.pact.pgse.crawler.finish.FinishedReportedProcessor;
import edu.hit.pact.pgse.crawler.store.FileSystemWebPageStore;
import edu.hit.pact.pgse.crawler.store.WebPageStore;
import edu.hit.pact.pgse.crawler.util.Anchor;
import edu.hit.pact.pgse.crawler.util.DuplicateUrlFilter;

import java.io.*;

import edu.hit.pact.pgse.util.*;

public class BBSCrawler extends Crawler{
	
	private BBSUnit bbsUnit;
	private DuplicateUrlFilter duplicateUrlFilter;
	private  final int THREADNUM = 4;
	ConcurrentLinkedQueue<Anchor> articleAnchorQueue = new ConcurrentLinkedQueue<Anchor>();
	private static int taskNo = 1;
	
	public BBSCrawler(RunnableTask runnableTask, FinishedReportedProcessor finishedReporter, boolean ifSend, boolean ifStore) {
		super(runnableTask, finishedReporter, LogFactory.getLog(BBSCrawler.class), ifSend, ifStore);
	}
	
	@Override
	public void start() throws InterruptedException, MalformedURLException {
		
		logger.info("bbs crawler begin to work.");
		
		this.bbsUnit = ((BBSTaskPiece) pieceProperty.getPiece()).getBbsUnit();
		this.duplicateUrlFilter = new DuplicateUrlFilter();

		URL homepage = bbsUnit.getUrl();
		TianYaBBSURL topicUrl = new TianYaBBSURL(homepage);
		topicUrl.fllter();
		String subitem = topicUrl.getSubitem();
		logger.info(homepage.toString());
		AtomicBoolean isFinished = new AtomicBoolean(false);
//		Anchor ch = new Anchor();
//		URL tu = new URL("http://www.tianya.cn/publicforum/content/worldlook/1/200740.shtml");
//		ch.setText("bbbbbbbbbbbbbbbb");
//		ch.setUrl(tu);
//		articleAnchorQueue.add(ch);
		AbstractBBSFetchorThread indexThread = new BBSCrawlerIndexFetchProcessor(
				logger, this.fetcher, this.store, articleAnchorQueue,"task_" + taskNo+"_indexThread", homepage,isFinished,true);
		indexThread.start();
		
		ConcurrentArrayList<ArticleUpdateInfo> updateList = new ConcurrentArrayList<ArticleUpdateInfo>();

		List<AbstractBBSFetchorThread> articleThreads = new ArrayList<AbstractBBSFetchorThread>();
		
		for (int i = 0; i < this.THREADNUM; i++) {
			WebPageFetcher articleFetcher = buildWebPageFetcher();
			WebPageStore articleStore = new FileSystemWebPageStore(ifStore);
			AbstractBBSFetchorThread articleFetchor = 
				new BBSArticleFetchProcessor(logger, articleFetcher, articleStore, articleAnchorQueue,
					                         "task_" + taskNo + "_articleThread_" + i, subitem,isFinished,updateList,true);
			
			articleThreads.add(articleFetchor);
		}
		taskNo ++;
        
		
		for (AbstractBBSFetchorThread thread : articleThreads) {
			thread.start();
		}
		
//		BBSArticleUpdateFetchor updateFetchor = new BBSArticleUpdateFetchor(logger,"task "+taskNo + "_updateFetchor",updateList);
//		updateFetchor.start();

//		indexThread.join();
//		for (AbstractBBSFetchorThread thread : articleThreads) {
//			thread.join();
//		}
		
		//startWithOneTopicPage();
	}

	
	
//	//only fetch one topic page
//	public void startWithOneTopicPage() throws InterruptedException, MalformedURLException {
//		URL url = bbsUnit.getUrl();
//		TianYaBBSURL topicUrl = new TianYaBBSURL(url);
//		topicUrl.fllter();
//		String subitem = topicUrl.getSubitem();
//		int pageno = topicUrl.getPageno();
//		int pageNum =0;
//		String path = Utilities.getDay()+"/" + url.getHost() + "/" + subitem + "/"; 
//		
//		while (pageNum++ < 100 && url != null) {
//			
//			logger.info("begin to download the topic page : " + pageno);
//			WebPageFetcher topicFetcher = buildWebPageFetcher();
//			this.bbsUnit.setUrl(url);
//			RawWebPage page = topicFetcher.fetchURL(url, 2, null, null, null);
//
//			if (page == null) {
//				pieceProperty.incrementFailedPageNum();
//				// pieceProperty.decrementCleanUrlQueueNum();
//				logger.error(url.toString() + " downs failed");
//				return;
//			}
//
//			List<String> interLinkList = page.getCommentLinks(); // to obtain
//																	// the all
//																	// internal
//																	// links
//
//			Queue<String> articleQueue = new LinkedList<String>();
//			boolean flag  = false;
//            List<URL> articleUrls = new ArrayList<URL>();
//			for (String interLink : interLinkList) {
//				URL tmpUrl = new URL(interLink);
//				if (bbsUnit.isArticleURL(tmpUrl,subitem)) {
//					articleUrls.add(tmpUrl);
//					
//					/*logger.info("begin article page:"+interLink);
//					GetComments getComments = new GetComments(path);
//					try {
//						getComments.GetBBS(interLink);
//					} catch (FailingHttpStatusCodeException e) {
//						e.printStackTrace();
//					} catch (IOException e) {
//						e.printStackTrace();
//					}
//					*/
//					
//					//fetchOneArticlePage(interLink,subitem);
//				} else if (bbsUnit.isNextPageUrl(tmpUrl,subitem,pageno)) {
//					logger.info("next page url : " + interLink);
//					url = tmpUrl;
//					flag = true;
//				}
//				else {
//					//logger.info("not tianya url " + interLink);
//				}
//			}
//			if(!flag)
//				url = null;
//			
//			
//			ArrayList<BBSArticleFetchProcessor> fetchorList = new ArrayList<BBSArticleFetchProcessor>();
//			
//			for(int threadno=0;threadno<this.THREADNUM;threadno++){
//				
//				int fromIndex = articleUrls.size() / this.THREADNUM * threadno ;
//				int toIndex = articleUrls.size() / this.THREADNUM * (threadno+1);
//				
//				if(threadno == this.THREADNUM-1){
//					toIndex = articleUrls.size();
//				}
//				
//				ArrayList<URL> urls = new ArrayList(articleUrls.subList(fromIndex, toIndex));
//				//fetchorList.add(new BBSArticleFetchProcessor(logger,"fethor "+threadno + " ",subitem,urls));
//			}
//			
//			for(BBSArticleFetchProcessor fetchor : fetchorList){
//				fetchor.start();
//			}
//			
//			for(BBSArticleFetchProcessor fetchor : fetchorList){
//				fetchor.join();
//			}
//			
//			
//			this.store.storeWebPage(page, subitem+"/",pageno,true);
//			
//			pageno ++;
//			
//			if(null != url){
//			logger.info("next page url is : " + url.toExternalForm());
//			}else{
//				logger.info("next page url is null");
//			}
//			
//		}
//		
//		logger.info("the topic("+subitem+") is completed.");
//		//finishedReporter.start();
//	}
//	
//	private void fetchOneArticlePage(String url,String subitem) throws InterruptedException, MalformedURLException {
//	    int NUM = 1;
//		logger.info("fetching " + NUM++ + url);
//		if (this.duplicateUrlFilter.contains(url)) {
//			return;
//		}
//		URL interLink = new URL(url);
//		RawWebPage articlePage = this.fetcher.fetchURL(interLink, 3, null, null, null);
//		if (articlePage == null) {
//			pieceProperty.incrementFailedPageNum();
//			logger.error(interLink.toString()+" downs failed");
//			return;
//		}
//		this.fetchedPageQueue.add(articlePage);
//		this.store.storeWebPage(articlePage,subitem);
//	}
	

	public void deactivate(){
		logger.info("closing bbs crawler");
		super.deactivate();
		this.bbsUnit = null;
		this.duplicateUrlFilter = null;
	}

}
